%% ensemble_test3(stegoMat, coverMat, modelFileName, ratio, TURN[, SUB_TURN])
% train model and then manually split training and testing sets.
% modelFileName: name of file to save model.
% ratio: nTrain/(nTrain+nTest)
%
function r = ensemble_test4(stegoMat, coverMat, modelFileName, ratio, TURN, SUB_TURN)
  %% initialize variables
   r.avg_time = 0;
   r.avg_err = 0;
   r.min_err = Inf;
   d_sub='automatic';
   L='automatic';
   sk = 0;
   sl = 0;
   if (nargin < 4)
       ratio = 0.5;
   end
   if (nargin < 6)
       SUB_TURN = 1;
   end
   if (nargin < 5)
       TURN = 1;
   end
   
   %% manually seperate training set and testing set
   disp('split sets:');
   [sM, sN] = size(stegoMat);
   [cM, cN] = size(coverMat);
   if (sN ~= cN)
       fprintf(2,'size of stegoMat %dx%d & coverMat %dx%d does not match',sM,sN,cM,cN);
       return;
   end
   sP = randperm(sM);
   cP = randperm(cM);
   M = floor(ratio * min(sM,cM));
   fprintf(2,'train set = %d\n',M*2);
   
   train_stegoMat = stegoMat(sP(1:M),:);
   test_stegoMat = stegoMat(sP(M+1:end),:);   
   train_coverMat = coverMat(cP(1:M),:);
   test_coverMat = coverMat(cP(M+1:end),:);
   
   tM = sM-M+cM-M;
   tP = randperm(tM);
   testMat = [test_stegoMat; test_coverMat];
   testMat = testMat(tP,:);
   fprintf(2,'test set = %d\n',tM);
   resultMat = [-1*ones(sM-M,1);+1*ones(cM-M,1)];
   resultMat = resultMat(tP,:);
   
   %% get model
   disp('train model');
   for i=1:TURN
      fprintf(1,'round %d\n',i);
      %% call ensemble
      r1=ensemble_wrap(train_stegoMat, train_coverMat,'same',d_sub,L,0.95);
      if i <= SUB_TURN
          sk = sk + r1.optimal.k;
          sl = sl + r1.optimal.L;
      end
      if i == SUB_TURN
          r.d_sub = round(sk / SUB_TURN);
          % r.L = round(sl / SUB_TURN);
          d_sub=r.d_sub
          % L=r.L
      end
      
      %% test
       returnMat = ensemble_predict(testMat, r1);
       p = nnz(returnMat - resultMat) / tM;
       fprintf(2,'tst_error = %0.4f\n',p);
       r1.test_error = p;
       r.avg_err =r.avg_err + r1.test_error;

      %% update return value
      if ~(r.min_err < r1.test_error)
          r.min_err = r1.test_error;
          r.best_model = r1;
      end
      
      r.avg_time=r.avg_time+r1.training_time;
   end
   r.avg_time=r.avg_time / TURN;
   r.avg_err= r.avg_err / TURN;
   
   %% save model
   disp('save model');
   model = r.best_model;
   save(modelFileName,'model');
end
